package lion.crawl;

import java.io.IOException;
import java.util.List;

import lion.crawl.parse.Page;
import lion.crawl.parse.URLRule;
import lion.crawl.rule.ParseRule;
import lion.crawl.util.HtmlUtils;
import lion.dev.lang.MapJ;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * 
 * 抓取器(包含抓取，解析等主要业务流程)
 * 
 * 抓取超时处理 当所有线程都在等待，并且一定时间内没有新的url时，视为抓取到头了
 * 
 * 抓取深度处理 <1为不限制，大于1为抓取循环深度 抓取内容处理 页面url再抓取处理
 * 
 * @author Helin
 * @date 2012-8-17
 * @email hl_0074@sina.com
 */
public class Crawler extends Thread {

	private Traveler travel;

	private boolean waiting;
	private boolean exit;

	private static Log log = LogFactory.getLog(Crawler.class);

	public Crawler(Traveler travel) {

		super();
		this.travel = travel;
	}

	@Override
	public void run() {

		// 1,获取url
		while (!exit) {
			Page page;
			waiting = true;
			try {
				page = travel.getUrlBase().get();
			} catch (Exception e1) {
				continue;
			}

			//
			if (page == null) {
				break;
			}

			waiting = false;
			if (page.isCrawled()) {
				continue;
			}
			// 2,解析内容
			String html = null;
			try {
				html = this.travel.getDownloader().curl(page.getUrl());
			} catch (IOException e) {
				this.travel.getParseListener().onParseError(page, e.getMessage());
				continue;
			} finally {
				travel.getUrlBase().crawled(page);
			}
			page.setCrawled(true);
			page.setContent(html);
			if (StringUtils.isBlank(html)) {
				this.travel.getParseListener().onParseError(page, "内容为空");
				continue;
			}
			try {
				parsePage(page);
			} catch (Exception e) {
				this.travel.getParseListener().onParseError(page, e.getMessage());
				continue;
			}
			// 3,解析页面url,添加url到地址库
			if (travel.getLevel() > 0 && page.getLevel() >= travel.getLevel()) {
				continue;
			}
			extractNewURLs(page);
		}
		log.debug("crawler #" + this.getId() + "  stoped");
	}

	public void exit() {

		this.exit = true;
		interrupt();
	}

	public boolean isWaiting() {

		return this.waiting;
	}

	private void extractNewURLs(Page page) {

		List<String> urls = HtmlUtils.findURLs(page.getUrl(), page.getContent());
		for (String url : urls) {
			for (URLRule r : travel.getPageRules()) {
				if (r.matches(url)) {
					travel.getUrlBase().add(new Page(url, page.getLevel() + 1));
					break;
				}
			}
		}
	}

	private void parsePage(Page page) {

		boolean needParse = false;
		for (URLRule r : travel.getPageRules()) {
			needParse = r.needParse(page.getUrl());
			if (needParse) {
				break;
			}
		}

		if (!needParse) { return; }

		MapJ result = new MapJ();
		this.travel.getParseListener().onParseBefore(page, result);
		for (ParseRule r : this.travel.getParseRules()) {
			r.extract(page.getContent(), result);
		}
		this.travel.getParseListener().onParseSuccess(page, result);
	}
}
